// Copyright (c) 2008-2010, Arne Claus
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the 
// following conditions are met:
//
// - Redistributions of source code must retain the above copyright notice, this list of conditions and the following 
//   disclaimer.
// - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 
//   disclaimer in the documentation and/or other materials provided with the distribution.
// - Neither the name Arne Claus nor the names of any contributors may be used to endorse or promote products
//   derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 
// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 
// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef __INCLUDED_CBL_SIMD_H__
#define __INCLUDED_CBL_SIMD_H__

#include "CBL_Defines.h"
#include "CBL_Math.h"

#ifdef CB_ARCH_X86
#if defined CB_USE_SSE4
	#define CB_USE_SSE3
	#include "smmintrin.h"

#elif defined CB_USE_SSE3
	#define CB_USE_SSE2
	#include "pmmintrin.h"
	
#elif defined CB_USE_SSE2
	#include "emmintrin.h"

#else
	#include "xmmintrin.h"
#endif
#endif

CB_SUBNAMESPACE_START(Math)

// =============================================================================================
//  Datatypes
// =============================================================================================

#ifdef CB_ARCH_X86
// =============================================================================================
//  SSE
// =============================================================================================

union Vec4f
{
	float Data[4];
	__m128 Packed;
};

union Vec4i
{
	int32 Data[4];
#ifdef CB_USE_SSE2
	__m128i Packed;
#endif
};

union Vec8s
{
	int16 Data[8];
#ifdef CB_USE_SSE2
	__m128i Packed;
#endif
};

union Vec16c
{
	int8 Data[16];
#ifdef CB_USE_SSE2
	__m128i Packed;
#endif
};

#elif defined(CB_ARCH_PPC)

// =============================================================================================
//  Altivec
// =============================================================================================

union Vec4f
{
	float Data[4];
	vector float Packed;
};

union Vec4i
{
	int Data[4];
	vector int Packed;
};

union Vec8s
{
	short Data[8];
	vector short Packed;
};

union Vec16c
{
	char Data[16];
	vector char Packed;
};


#else

// =============================================================================================
//	Unsupported architecture
// =============================================================================================

struct Vec4f
{
	float Data[4];	
};

struct Vec4i
{
	int32 Data[4];	
};

union Vec8s
{
	int16 Data[8];
};

union Vec16c
{
	int8 Data[16];
};

#endif

// Note:	Not every function is fully supported with all architectures. This is because the function
//			sets of SSE and Altivec differ in some ways. If a function is not supported by one
//			architecture a FPU fallback function will be used.
//			Also note that with classic SSE accelerated functions are only available for Vec4f, because
//			integer support was not added until SSE2 and was not extended to (nearly) full functionality
//			before SSE4.
//			As of this, there are hints before each function prototype to note which architectures are supported.
//			A star behind an architectures hints, that the function is not fully supported and is simulated with
//			one or more SIMD commands or with the use of FPU commands and one or more SIMD command.
//
//			By default only SSE(1) functionality is used. If you want to enable functionality from higher
//			SSE versions you can use the following preprocessor defines:
//			CB_USE_SSE2, CB_USE_SSE3, CB_USE_SSE4
//			Note, that there is no runtime check if the CPU supports those function. However, You can implement
//			such a check yourself, with the functions provided by CB_Cpu.h.
//			
//			SIMD Timetable:
//
//			Altivec:	PowerPC MPC7400 "G4" (1999)
//			SSE:		Intel Pentium3 "Katmai" (1999), Amd Athlon XP "Palomino" (2001)
//			SSE2:		Intel Pentium4 "Willamette" (2000), Amd Athlon 64 "Clawhammer" (2003)
//			SSE3:		Intel Pentium4 "Prescott" (2004), Amd Athlon 64 "Venice" (2005)
//			SSE4:		Intel Core2 "Penryn" (2007)
//
//			The functions provided here are NOT the full set of available SIMD functions. It's only a subset,
//			containing the most used SIMD functions for genereal purpose calculations.

// =============================================================================================
//  Vec4f declarations
// =============================================================================================

// SSE, Altivec
// out[i] = v
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat(Vec4f& _rOut, float _Value);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec*
// out[i] = v[s]
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat(Vec4f& _rOut, Vec4f& _rValues, uint32 _Selector);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] = v[s[i]]
CB_FORCEINLINE void CB_INLINE_ATTR VecSwizzle(Vec4f& _rOut, const Vec4f& _rValues, const Vec4i& _rSelector);

// ---------------------------------------------------------------------------------------------

// SSE
// out[i] = l[i/2 + 2], out[i+1] = r[i/2 + 2]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeBack(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE
// out[i] = l[i/2], out[i+1] = r[i/2]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeFront(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = l[i] + r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAdd(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = l[i] - r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecSub(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = l[i] * r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMul(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec*
// out[i] = l[i] / r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecDiv(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = l[i] * r[i] + a[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMAdd(Vec4f& _rOut, const Vec4f& _rMLhs, const Vec4f& _rMRhs, const Vec4f& _rAdd);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = -(l[i] * r[i] - a[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecNMSub(Vec4f& _rOut, const Vec4f& _rMLhs, const Vec4f& _rMRhs, const Vec4f& _rSub);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = l[i] * s
CB_FORCEINLINE void CB_INLINE_ATTR VecScale(Vec4f& _rOut, const Vec4f& _rLhs, float _Scale);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = min(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMin(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = max(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMax(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec*
// out[i] = sqrt(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecSqrt(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = 1 / sqrt(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecRESqrt(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// SSE, Altivec
// out[i] = 1 / v[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecRE(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = abs(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecAbs(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = floor(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecFloor(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = ceil(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecCeil(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = round(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecRound(Vec4f& _rOut, const Vec4f& _rValues);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] == v
bool VecAll(const Vec4f& _rValues, float _Value);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] == r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpEQ(Vec4i& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] > r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGT(Vec4i& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] < r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLT(Vec4i& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] >= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGE(Vec4i& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] <= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLE(Vec4i& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] == r[i]) ? !0 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpEQN(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] > r[i]) ? !0 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGTN(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] < r[i]) ? !0 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLTN(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] >= r[i]) ? !0 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGEN(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE*, Altivec
// out[i] = (l[i] <= r[i]) ? !0 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLEN(Vec4f& _rOut, const Vec4f& _rLhs, const Vec4f& _rRhs);

// =============================================================================================
//  Vec4i declarations
// =============================================================================================

// SSE2, Altivec
// out[i] = v
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat(Vec4i& _rOut, int32 _Value);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec*
// out[i] = v[s]
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat(Vec4f& _rOut, Vec4i& _rValues, uint32 _Selector);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] = v[s[i]]
CB_FORCEINLINE void CB_INLINE_ATTR VecSwizzle(Vec4i& _rOut, const Vec4i& _rValues, const Vec4i& _rSelector);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i/2 + 2], out[i+1] = r[i/2 + 2]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeBack(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i/2], out[i+1] = r[i/2]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeFront(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] + r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAdd(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] - r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecSub(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] * r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMul(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] / r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecDiv(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec*
// out[i] = l[i] * r[i] + a[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMAdd(Vec4i& _rOut, const Vec4i& _rMLhs, const Vec4i& _rMRhs, const Vec4i& _rAdd);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] * s
CB_FORCEINLINE void CB_INLINE_ATTR VecScale(Vec4i& _rOut, const Vec4i& _rLhs, int32 _Scale);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = min(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMin(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = max(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMax(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] & r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAnd(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] & !r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAndNot(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] | r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecOr(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] ^ r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecXOr(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i] << r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecLShift(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i] >> r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecRShift(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE4*, Altivec
// out[i] = abs(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecAbs(Vec4i& _rOut, const Vec4i& _rValues);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] == v
bool VecAll(const Vec4i& _rValues, int32 _Value);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] == r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpEQ(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] > r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGT(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] < r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLT(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec
// out[i] = (l[i] >= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGE(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec
// out[i] = (l[i] <= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLE(Vec4i& _rOut, const Vec4i& _rLhs, const Vec4i& _rRhs);

// =============================================================================================
//  Vec8s declarations
// =============================================================================================

// SSE2, Altivec
// out[i] = v
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat(Vec8s& _rOut, int16 _Value);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec*
// out[i] = v[s]
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat(Vec8s& _rOut, Vec8s& _rValues, uint32 _Selector);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] = v[s[i]]
CB_FORCEINLINE void CB_INLINE_ATTR VecSwizzle(Vec8s& _rOut, const Vec8s& _rValues, const Vec8s& _rSelector);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i/2 + 4], out[i+1] = r[i/2 + 4]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeBack(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i/2], out[i+1] = r[i/2]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeFront(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] + r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAdd(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] - r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecSub(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] * r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMul(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] / r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecDiv(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec*
// out[i] = l[i] * r[i] + a[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMAdd(Vec8s& _rOut, const Vec8s& _rMLhs, const Vec8s& _rMRhs, const Vec8s& _rAdd);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] * s
CB_FORCEINLINE void CB_INLINE_ATTR VecScale(Vec8s& _rOut, const Vec8s& _rLhs, int16 _Scale);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = min(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMin(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = max(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMax(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] & r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAnd(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] & !r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAndNot(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] | r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecOr(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] ^ r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecXOr(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i] << r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecLShift(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2
// out[i] = l[i] >> r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecRShift(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE4*, Altivec
// out[i] = abs(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecAbs(Vec8s& _rOut, const Vec8s& _rValues);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] == v
bool VecAll(const Vec8s& _rValues, int16 _Value);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] == r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpEQ(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] > r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGT(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] < r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLT(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec
// out[i] = (l[i] >= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGE(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec
// out[i] = (l[i] <= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLE(Vec8s& _rOut, const Vec8s& _rLhs, const Vec8s& _rRhs);

// =============================================================================================
//  Vec16c declarations
// =============================================================================================

// SSE2, Altivec
// out[i] = v
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat		(Vec16c& _rOut, int8 _Value);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec*
// out[i] = v[s]
CB_FORCEINLINE void CB_INLINE_ATTR VecSplat		(Vec16c& _rOut, Vec16c& _rValues, uint32 _Selector);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] = v[s[i]]
CB_FORCEINLINE void CB_INLINE_ATTR VecSwizzle		(Vec16c& _rOut, const Vec16c& _rValues, const Vec16c& _rSelector);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i/2 + 8], out[i+1] = r[i/2 + 8]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeBack	(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i/2], out[i+1] = r[i/2]
CB_FORCEINLINE void CB_INLINE_ATTR VecMergeFront	(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] + r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAdd			(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] - r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecSub			(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] * r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMul			(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i] / r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecDiv			(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec*
// out[i] = l[i] * r[i] + a[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecMAdd		(Vec16c& _rOut, const Vec16c& _rMLhs, const Vec16c& _rMRhs, const Vec16c& _rAdd);

// ---------------------------------------------------------------------------------------------

//
// out[i] = l[i]* s
CB_FORCEINLINE void CB_INLINE_ATTR VecScale		(Vec16c& _rOut, const Vec16c& _rLhs, int8 _Scale);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = min(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMin			(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE4, Altivec
// out[i] = max(l[i], r[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecMax(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] & r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAnd(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] & !r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecAndNot(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] | r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecOr(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = l[i] ^ r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecXOr(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] = l[i] << r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecLShift(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] = l[i] >> r[i]
CB_FORCEINLINE void CB_INLINE_ATTR VecRShift(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE4*, Altivec
// out[i] = abs(v[i])
CB_FORCEINLINE void CB_INLINE_ATTR VecAbs(Vec16c& _rOut, const Vec16c& _rValues);

// ---------------------------------------------------------------------------------------------

// Altivec
// out[i] == v
bool VecAll(const Vec16c& _rValues, int8 _Value);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] == r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpEQ(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] > r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGT(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2, Altivec
// out[i] = (l[i] < r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLT(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec
// out[i] = (l[i] >= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpGE(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

// ---------------------------------------------------------------------------------------------

// SSE2*, Altivec
// out[i] = (l[i] <= r[i]) ? 1 : 0
CB_FORCEINLINE void CB_INLINE_ATTR VecCmpLE(Vec16c& _rOut, const Vec16c& _rLhs, const Vec16c& _rRhs);

#if defined(CB_SIMD_USE_FALLBACK)
#include "CBL_SIMD_Fallback.hpp"
#elif defined(CB_ARCH_X86)
#include "CBL_SIMD_SSE.hpp"
#elif defined(CB_ARCH_PPC)
#include "CBL_SIMD_Altivec.hpp"
#else
#include "CBL_SIMD_Fallback.hpp"
#endif

CB_SUBNAMESPACE_END

#endif
